# =============================================================== #
# Reproducibility of:											                        #
# Drivers and Trends for the Equality of Opportunity for Sexual   #
# and Gender Minorities: A Panel Approach Equality of Opportunity #
# for Sexual and Gender Minorities 2024							              #
# 																                                #
# Code written by Omar Alburqueque and reviewed by Paola Ballon   #
# Contact: oalburquequechav@worldbank.org, pballon@worldbank.org  #
# =============================================================== #

# ----------------------------------------------- #
# Data extraction from KOF Globalisation Database #
# ----------------------------------------------- #

library(tsdbapi)
library(purrr)
library(dplyr)
library(tidyr)
library(lubridate)
library(xts)
library(writexl)

project_root     <- getwd()
intermediate_dir <- file.path(project_root, "intermediate files")

if (!dir.exists(intermediate_dir)) {
  dir.create(intermediate_dir, recursive = TRUE)
}

# The KOF Globalisation Index collection is publicly accessible.
# No API key is required when access_type is set to "public".
tsdbapi::set_config(access_type = "public")

# By default, use the latest released vintage available on the run date.
# For strict reproducibility, replace Sys.Date() with a fixed date.
vintage_date <- Sys.Date()

# List of ISO3 country codes in lowercase
# CAUTION: (1) Timor-Leste: TLS -> TMP
# No data for Kosovo
countries <- c(
  "dza", "arg", "arm", "bgd", "btn", "bra", "khm", "cmr", "can", "chl",
  "chn", "cri", "civ", "dji", "ecu", "egy", "eth", "fji", "fra", "geo",
  "deu", "gha", "gnb", "guy", "hti", "hnd", "ind", "idn", "irq", "isr",
  "jam", "jpn", "jor", "ken", "kor", "ksv", "kgz", "lbn", "mus", "mex",
  "mng", "mar", "moz", "npl", "nzl", "nga", "nor", "pak", "png", "phl",
  "srb", "zaf", "esp", "lka", "sdn", "tza", "tha", "tmp", "tun", "tur",
  "ukr", "ury", "vnm", "zwe"
)

# Indicators to fetch
indicators <- c(
  "gi", "gidf", "gidj", "cugi", "cugidf", "cugidj", "ecgi", "ecgidf", "ecgidj",
  "figi", "figidf", "figidj", "ingi", "ingidf", "ingidj", "ipgi", "ipgidf", "ipgidj",
  "pogi", "pogidf", "pogidj", "sogi", "sogidf", "sogidj", "trgi", "trgidf", "trgidj"
)

# Construct all requested time-series keys
requested_series <- tidyr::crossing(
  country = countries,
  indicator = indicators
) %>%
  mutate(ts_key = paste0("ch.kof.globidx.v2020.", indicator, ".", country))

# Download in batches to avoid one very large request
batch_size <- 200L
key_batches <- split(
  requested_series$ts_key,
  ceiling(seq_along(requested_series$ts_key) / batch_size)
)

results_by_batch <- map(seq_along(key_batches), function(i) {
  cat("Processing batch", i, "out of", length(key_batches), "\n")
  
  tryCatch(
    tsdbapi::read_ts(
      ts_keys = key_batches[[i]],
      valid_on = vintage_date,
      ignore_missing = TRUE
    ),
    error = function(e) {
      warning("Batch ", i, "cannot be downloaded: ", conditionMessage(e))
      list()
    }
  )
})

# Combine the named lists returned by each batch
results <- do.call(c, results_by_batch)

if (length(results) == 0L) {
  stop("KOF did not return any requested time series.")
}

# Record requested series that were not returned
no_data <- requested_series %>%
  filter(!ts_key %in% names(results))

# Convert each ts/xts object to a common long-format data frame
series_to_df <- function(serie, ts_key) {
  key_parts <- strsplit(ts_key, ".", fixed = TRUE)[[1]]
  indicator <- key_parts[length(key_parts) - 1L]
  country   <- key_parts[length(key_parts)]
  
  if (inherits(serie, "xts")) {
    year  <- lubridate::year(xts::index(serie))
    value <- as.numeric(xts::coredata(serie))
  } else if (inherits(serie, "ts")) {
    year  <- as.integer(round(as.numeric(stats::time(serie))))
    value <- as.numeric(serie)
  } else {
    stop("Unsupported object class for ", ts_key)
  }
  
  tibble(
    country = country,
    indicator = indicator,
    year = year,
    value = value
  )
}

final_df <- purrr::imap_dfr(results, series_to_df)

# Reshape to wide format: one column per indicator
final_wide <- imap_dfr(results, series_to_df) %>%
  pivot_wider(
    names_from = indicator,
    values_from = value
  ) %>%
  filter(if_any(-c(country, year), ~ !is.na(.x))) %>%
  mutate(
    country = toupper(country),
    country = if_else(country == "TMP", "TLS", country)
  ) %>%
  arrange(country, year)

write_xlsx(
  final_wide,
  file.path(intermediate_dir, "kof_data.xlsx")
)
